Prepare input data

Remove “PartyDetailsDefinitionRequest” Messages (they all have duplicate arrival times and null processing time)
Normalize event arrival_T at $T_0 = 0$ Method: $T_i = T_i - T_0 \mbox{ for } i=0,...,n$.
Spread out messages with duplicate arrival times; Method: $T_i = T_i + k \mbox{ for } i = 0,...,n \mbox{ for } k=0..d$ where $d = $ number of duplicates at $T_i$
Calculate inter-message delta_t in nanoseconds
Convert latency/processing time PNQM Latency (ms) to duration in nanoseconds
Use Msg Type as new factor type
Calculate event stop time, departure_T, in nanoseconds
Number events in N

input_limit <- 1.2e11 # Two minutes in ns

event_data <- .msgw_in_import %>%
  filter( !`Msg Type` %in% c( "PartyDetailsDefinitionRequest" ) ) %>%
  { if( input_limit ) filter(  ., ts_ns < first(ts_ns) + input_limit ) } %>%
  mutate(
    t = ts_ns - first(ts_ns),
    duration = as.numeric( na_if( `PNQM Latency (ms)`, "null" ) ) * 1e6,
    type = as.factor( `Msg Type` )
  ) %>%
  group_by( t ) %>%
  mutate(
    dup_count = n() - 1,
    dup_number = row_number() - 1
  ) %>%
  select( t, duration, type, dup_count, dup_number ) %>%
  ungroup() %>%
  arrange( t ) %>%
  mutate(
    N = row_number(),
    arrival_T = t + dup_number,
    delta_t = arrival_T - lag(arrival_T),
    departure_T = arrival_T + duration
  ) %>%
  select( N, arrival_T, delta_t, type, duration, departure_T )

event_data

Successfully loaded 59498 observations from the first $input_limit$ nanoseconds of our event data.

Vizualizations concerning Event Arrival Time

hist( as.numeric( event_data$arrival_T ), breaks=120, col="blue", xlab = "Event Time in ns over the first two minutes of data.", main="Histogram of Event Freq per Sec" )

par(mfrow=c( 4, 1 ) )#, mai=c(0.1, 0.1, 0.1, 0.1) )
stripchart( as.numeric( head( event_data$arrival_T, 10000 ) ), pch=19, cex=0.5, col="blue", xlab="Event Time in ns", main="1-D Stripchart of Event Time; first 10000 observations." )
stripchart( as.numeric( head( event_data$arrival_T, 1000 ) ), pch=19, cex=0.5, col="blue", xlab="Event Time in ns", main="1-D Stripchart of Event Time; first 1000 observations." )
stripchart( as.numeric( head( event_data$arrival_T, 100 ) ), pch=19, cex=0.5, col="blue", xlab="Event Time in ns", main="1-D Stripchart of Event Time; first 100 observations." )
stripchart( as.numeric( head( event_data$arrival_T, 10 ) ), pch=19, cex=0.5, col="blue", xlab="Event Time in ns", main="1-D Stripchart of Event Time; first 10 observations." )

par( mfrow=c(1, 1) )

plot( head( event_data$arrival_T, 10000 ), head( event_data$delta_t, 10000 ), col="blue", pch=19, xlab="Event Time in ns", ylab="Inter-Event Delta_t in ns", main="Inter-Event Delta_t against Event Time" )

plot( head( event_data$arrival_T, 1000 ), head( event_data$delta_t, 1000 ), col="blue", pch=19, xlab="Event Time in ns", ylab="Inter-Event Delta_t in ns", main="Inter-Event Delta_t against Event Time" )

plot( head( event_data$arrival_T, 100 ), head( event_data$delta_t, 100 ), col="blue", pch=19, xlab="Event Time in ns", ylab="Inter-Event Delta_t in ns", main="Inter-Event Delta_t against Event Time" )

plot( head( event_data$arrival_T, 10 ), head( event_data$delta_t, 10 ), col="blue", pch=19, xlab="Event Time in ns", ylab="Inter-Event Delta_t in ns", main="Inter-Event Delta_t against Event Time" )

Visualizations concering inter-event delta_t

. <- event_data %>%
  #slice( -1 ) %>%
  mutate( delta_t = replace_na( as.numeric( delta_t ), 0 ) ) %>%
  select( delta_t )
delta_t = .$delta_t

str(delta_t)

##  num [1:59498] 0 588 69017 9380 1689075 ...

summary(delta_t)

##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
##         0      8480     90968   2016875    949788 102475959

describe(delta_t)

acf(delta_t)

pacf(delta_t)

plot( event_data$arrival_T, delta_t, col="blue", pch=19, xlab="Event Time in ns", ylab="Inter-Event Delta_t in ns", main="Inter-Event Delta_t against Event Time" )

plot( head( delta_t, 1000 ), head( event_data$duration, 1000 ), pch=19, col="blue", xlab="Inter-Event Delta_t", ylab="Duration of Event in ns", main="Event Duration against Inter-Event Delta_t" )

hist( as.numeric( delta_t), col="blue", main="Histogram of Inter-event Delta_t", xlab="Delta_t in Nanoseconds")

hist( log( as.numeric( delta_t)), col="blue", main="Histogram of log( Inter-event Delta_t )", xlab="log(Delta_t) in Nanoseconds")

{
  qqnorm( as.numeric( delta_t ) )
  qqline(as.numeric(delta_t), distribution=qnorm)
}

{
  qqplot(x=qexp(ppoints(1000)), y=as.numeric(delta_t), main="Exponential Q-Q Plot",
         xlab="Theoretical Quantiles", ylab= "Delta_t Quantiles")
  qqline(as.numeric(delta_t), distribution=qexp)
}

Visualizations concering the Number of Concurrent Events over Time (Queueing Process)

in_queue <- event_data %>%
  filter( !is.na( departure_T ), departure_T != 0 ) %>%
  mutate( t = arrival_T, step = 1 ) %>%
  select( t, step )

out_queue <- event_data %>%
  filter( !is.na( departure_T ), departure_T != 0 ) %>%
  mutate( t = departure_T, step = -1 ) %>%
  select( t, step )

queue <- bind_rows( in_queue, out_queue ) %>%
  arrange( t ) %>%
  mutate( size = cumsum( step ))
  
plot( queue$t, queue$size, type="l", col="blue", pch=19, cex=0.5, main="Concurrent Event Count at Time t.", xlab='Time t in Nanoseconds', ylab="Concurrent event count" )

hist( log( queue$size ), col="blue", main="Histogram of log( Concurrent Event Count )", xlab="log( Concurrent Event Count )")

Data Prep

Import Data from File

Import message data from csv source.

Prepare input data

Vizualizations concerning Event Arrival Time

Visualizations concering inter-event delta_t

Visualizations concering the Number of Concurrent Events over Time (Queueing Process)